
import requests
import json
from dbutil.pgsql import PgsqlPipeline
from datetime import date

headers = {
    'Cookie': 'ASP.NET_SessionId=citapgvmwpvq4qscezfwqjwr; NTKF_T2D_CLIENTID=guest8EE3B5EE-2EEE-0022-B9B6-F4D65DCFD295; nTalk_CACHE_DATA={uid:kf_9255_ISME9754_guest8EE3B5EE-2EEE-00,tid:1567475785166484}; sc_ext_session=kr5rhbkw4ldkmlstifvmj152; nissan#lang=zh-CN; SC_ANALYTICS_GLOBAL_COOKIE=5b20d0477cfd4924913d8cce649b25ab|True; sc_ext_contact=5b20d0477cfd4924913d8cce649b25ab|True; NO_PAGE_DURATION=2019/9/3 21:18:07; no_screen=1280%7C800; Place=%7B%22province%22%3A%22%E6%B2%B3%E5%8C%97%22%2C%22city%22%3A%22%E7%9F%B3%E5%AE%B6%E5%BA%84%22%7D',
    'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.132 Safari/537.36',
}

#读取cities.json文件取出省，市
def get_data():
    with open("./dbutil/cities.json", 'r') as load_f:
        data_list = json.load(load_f)['provinces']
        return data_list


#post请求 获取json文件
def get_Dealers(citysName):
    url = 'https://www.dongfeng-nissan.com.cn/Nissan/ajax/Distributor/GetJsonDistributorList'

    data = {'city': citysName}

    response = requests.post(url, headers=headers, data=data, verify=False)
    dealers = json.loads(response.text)['data']['DealerInfos']
    return dealers

#解析json文件

def get_item(dealer, province_Name, citysName):
    item = {
        "dealer_name": dealer["StoreName"],
        "brand_id": None,
        "address": dealer["Address"],
        "brand": "日产",
        "province": province_Name,
        "city": citysName,
        "sale_call": dealer["SaleTel"],
        "customer_service_call": dealer["ServiceTel"],
        "update_time": date.today(),
        "longitude": dealer['Longitude'],
        "latitude": dealer['Latitude'],
        "dealer_type": None,
        "manufacturer_id": None,
        "manufacturer": "东风日产",
        "state": None,
        "opening_date": None,
        "close_date": None,
        "dealer_id_web": None,
        "controlling_shareholder": None,
        "other_shareholders": None,
        "status": None,
        "remarks": None,

    }
    return item

#实例化函数返回items
def get_items():
    items = []
    data_list = get_data()
    for data in data_list:
        # print(data)
        province_Name = data['provinceName']
        citys = data['citys']
        for city in citys:
            citysName = city['citysName']
            dealers = get_Dealers(citysName)
            for dealer in dealers:
                item = get_item(dealer, province_Name, citysName)
                items.append(item)
    return items


#主进程存储
def main():
    print('爬虫开始--------->')
    items = get_items()
    print('数据存储')
    pg = PgsqlPipeline()
    for item in items:
        pg.process_item(item)
    pg.close()
    print('数据存储完成')

if __name__ == '__main__':
    main()
